{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Forward  Stepwise Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:12:38.677971Z",
     "start_time": "2023-10-07T12:12:38.276277Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import r2_score,adjusted_rand_score\n",
    "\n",
    "import statsmodels.api as sm\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:12:38.694701Z",
     "start_time": "2023-10-07T12:12:38.680541Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(400, 11)\n"
     ]
    },
    {
     "data": {
      "text/plain": "    Income  Limit  Rating  Cards  Age  Education  Gender Student Married  \\\n1   14.891   3606     283      2   34         11    Male      No     Yes   \n2  106.025   6645     483      3   82         15  Female     Yes     Yes   \n3  104.593   7075     514      4   71         11    Male      No      No   \n4  148.924   9504     681      3   36         11  Female      No      No   \n5   55.882   4897     357      2   68         16    Male      No     Yes   \n\n   Ethnicity  Balance  \n1  Caucasian      333  \n2      Asian      903  \n3      Asian      580  \n4      Asian      964  \n5  Caucasian      331  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Income</th>\n      <th>Limit</th>\n      <th>Rating</th>\n      <th>Cards</th>\n      <th>Age</th>\n      <th>Education</th>\n      <th>Gender</th>\n      <th>Student</th>\n      <th>Married</th>\n      <th>Ethnicity</th>\n      <th>Balance</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>14.891</td>\n      <td>3606</td>\n      <td>283</td>\n      <td>2</td>\n      <td>34</td>\n      <td>11</td>\n      <td>Male</td>\n      <td>No</td>\n      <td>Yes</td>\n      <td>Caucasian</td>\n      <td>333</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>106.025</td>\n      <td>6645</td>\n      <td>483</td>\n      <td>3</td>\n      <td>82</td>\n      <td>15</td>\n      <td>Female</td>\n      <td>Yes</td>\n      <td>Yes</td>\n      <td>Asian</td>\n      <td>903</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>104.593</td>\n      <td>7075</td>\n      <td>514</td>\n      <td>4</td>\n      <td>71</td>\n      <td>11</td>\n      <td>Male</td>\n      <td>No</td>\n      <td>No</td>\n      <td>Asian</td>\n      <td>580</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>148.924</td>\n      <td>9504</td>\n      <td>681</td>\n      <td>3</td>\n      <td>36</td>\n      <td>11</td>\n      <td>Female</td>\n      <td>No</td>\n      <td>No</td>\n      <td>Asian</td>\n      <td>964</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>55.882</td>\n      <td>4897</td>\n      <td>357</td>\n      <td>2</td>\n      <td>68</td>\n      <td>16</td>\n      <td>Male</td>\n      <td>No</td>\n      <td>Yes</td>\n      <td>Caucasian</td>\n      <td>331</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(r'../data/Credit.csv',index_col=0)\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:12:38.720611Z",
     "start_time": "2023-10-07T12:12:38.695544Z"
    }
   },
   "outputs": [],
   "source": [
    "gender_encoding = {'Male':1,'Female':0}\n",
    "ethnicity = pd.get_dummies(data['Ethnicity'],drop_first=True)\n",
    "yes_no_encoding = {'Yes':1,'No':0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:12:38.739929Z",
     "start_time": "2023-10-07T12:12:38.701469Z"
    }
   },
   "outputs": [],
   "source": [
    "data['Gender'] = data['Gender'].map(gender_encoding)\n",
    "data['Student'] = data['Student'].map(yes_no_encoding)\n",
    "data['Married'] = data['Married'].map(yes_no_encoding)\n",
    "data.drop('Ethnicity',axis = 1,inplace = True)\n",
    "data = pd.concat([data,ethnicity],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:12:38.757099Z",
     "start_time": "2023-10-07T12:12:38.706706Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "    Income  Limit  Rating  Cards  Age  Education  Gender  Student  Married  \\\n1   14.891   3606     283      2   34         11       1        0        1   \n2  106.025   6645     483      3   82         15       0        1        1   \n3  104.593   7075     514      4   71         11       1        0        0   \n4  148.924   9504     681      3   36         11       0        0        0   \n5   55.882   4897     357      2   68         16       1        0        1   \n\n   Balance  Asian  Caucasian  \n1      333  False       True  \n2      903   True      False  \n3      580   True      False  \n4      964   True      False  \n5      331  False       True  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Income</th>\n      <th>Limit</th>\n      <th>Rating</th>\n      <th>Cards</th>\n      <th>Age</th>\n      <th>Education</th>\n      <th>Gender</th>\n      <th>Student</th>\n      <th>Married</th>\n      <th>Balance</th>\n      <th>Asian</th>\n      <th>Caucasian</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>14.891</td>\n      <td>3606</td>\n      <td>283</td>\n      <td>2</td>\n      <td>34</td>\n      <td>11</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>333</td>\n      <td>False</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>106.025</td>\n      <td>6645</td>\n      <td>483</td>\n      <td>3</td>\n      <td>82</td>\n      <td>15</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>903</td>\n      <td>True</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>104.593</td>\n      <td>7075</td>\n      <td>514</td>\n      <td>4</td>\n      <td>71</td>\n      <td>11</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>580</td>\n      <td>True</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>148.924</td>\n      <td>9504</td>\n      <td>681</td>\n      <td>3</td>\n      <td>36</td>\n      <td>11</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>964</td>\n      <td>True</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>55.882</td>\n      <td>4897</td>\n      <td>357</td>\n      <td>2</td>\n      <td>68</td>\n      <td>16</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>331</td>\n      <td>False</td>\n      <td>True</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:12:38.757245Z",
     "start_time": "2023-10-07T12:12:38.709811Z"
    }
   },
   "outputs": [],
   "source": [
    "features = [col for col in data.columns if not col == 'Balance']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:12:38.757297Z",
     "start_time": "2023-10-07T12:12:38.713392Z"
    }
   },
   "outputs": [],
   "source": [
    "def forward_stepwise_selection(data,target):\n",
    "    total_features = [[]]\n",
    "    score_dict = {}\n",
    "    remaining_features = [col for col in data.columns if not col == target]\n",
    "    for i in range(1,len(data.columns)):\n",
    "        best_score = 0;best_feature = None\n",
    "        for feature in remaining_features:\n",
    "\n",
    "            X = total_features[i-1] + [feature]\n",
    "            model = LinearRegression().fit(data[X],data[target])\n",
    "            score = r2_score(data[target],model.predict(data[X]))\n",
    "#             print('For len {}, feature - {}, score is {}'.format(i,feature,score))\n",
    "\n",
    "            if score > best_score:\n",
    "                best_score = score\n",
    "                best_feature = feature\n",
    "        total_features.append(total_features[i-1] + [best_feature])\n",
    "        remaining_features.remove(best_feature)\n",
    "        score_dict[i] = best_score\n",
    "    return total_features,score_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:12:38.954706Z",
     "start_time": "2023-10-07T12:12:38.744475Z"
    }
   },
   "outputs": [],
   "source": [
    "total_predictors,score_dict = forward_stepwise_selection(data,'Balance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-07T12:12:39.301860Z",
     "start_time": "2023-10-07T12:12:38.821129Z"
    }
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "__init__() got an unexpected keyword argument 'size'",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mTypeError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[9], line 3\u001B[0m\n\u001B[1;32m      1\u001B[0m temp \u001B[38;5;241m=\u001B[39m pd\u001B[38;5;241m.\u001B[39mDataFrame({\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mNumber of predictors\u001B[39m\u001B[38;5;124m'\u001B[39m:\u001B[38;5;28mlist\u001B[39m(score_dict\u001B[38;5;241m.\u001B[39mkeys()),\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mR2_Score\u001B[39m\u001B[38;5;124m'\u001B[39m:\u001B[38;5;28mlist\u001B[39m(score_dict\u001B[38;5;241m.\u001B[39mvalues())})\n\u001B[1;32m      2\u001B[0m plt\u001B[38;5;241m.\u001B[39mfigure(figsize \u001B[38;5;241m=\u001B[39m (\u001B[38;5;241m12\u001B[39m,\u001B[38;5;241m6\u001B[39m))\n\u001B[0;32m----> 3\u001B[0m g \u001B[38;5;241m=\u001B[39m \u001B[43msns\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mFacetGrid\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdata\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mtemp\u001B[49m\u001B[43m,\u001B[49m\u001B[43msize\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;241;43m5\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m      4\u001B[0m g\u001B[38;5;241m.\u001B[39mmap(plt\u001B[38;5;241m.\u001B[39mscatter, \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mNumber of predictors\u001B[39m\u001B[38;5;124m'\u001B[39m , \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mR2_Score\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[1;32m      5\u001B[0m g\u001B[38;5;241m.\u001B[39mmap(plt\u001B[38;5;241m.\u001B[39mplot, \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mNumber of predictors\u001B[39m\u001B[38;5;124m'\u001B[39m, \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mR2_Score\u001B[39m\u001B[38;5;124m'\u001B[39m)\n",
      "\u001B[0;31mTypeError\u001B[0m: __init__() got an unexpected keyword argument 'size'"
     ]
    },
    {
     "data": {
      "text/plain": "<Figure size 1200x600 with 0 Axes>"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "temp = pd.DataFrame({'Number of predictors':list(score_dict.keys()),'R2_Score':list(score_dict.values())})\n",
    "plt.figure(figsize = (12,6))\n",
    "g = sns.FacetGrid(data = temp,size=5)\n",
    "g.map(plt.scatter, 'Number of predictors' , 'R2_Score')\n",
    "g.map(plt.plot, 'Number of predictors', 'R2_Score')\n",
    "plt.title('Forward Selection')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-07T12:12:39.232077Z"
    }
   },
   "outputs": [],
   "source": [
    "for i,feature in enumerate(total_predictors):\n",
    "    print('Subset of size {} is'.format(i), feature)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Choosing the best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-07T12:12:39.232758Z"
    }
   },
   "outputs": [],
   "source": [
    "score_dict = {}\n",
    "for i,feature in enumerate(total_predictors[1:]):\n",
    "    X = data[feature]\n",
    "    y = data['Balance']\n",
    "    X = sm.add_constant(X)\n",
    "    result = sm.OLS(y, X).fit()\n",
    "    score_dict[i+1] = result.rsquared_adj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-07T12:12:39.234177Z"
    }
   },
   "outputs": [],
   "source": [
    "temp = pd.DataFrame({'Number of predictors':list(score_dict.keys()),'Adjusted_R2_Score':list(score_dict.values())})\n",
    "plt.figure(figsize = (12,6))\n",
    "g = sns.FacetGrid(data = temp,size=5)\n",
    "g.map(plt.scatter, 'Number of predictors' , 'Adjusted_R2_Score')\n",
    "g.map(plt.plot, 'Number of predictors', 'Adjusted_R2_Score')\n",
    "plt.title('Chossing the best model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-07T12:12:39.235393Z"
    }
   },
   "outputs": [],
   "source": [
    "score_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-07T12:12:39.236293Z"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
